from glob import glob
import  pandas as pd
from  tqdm import tqdm

voc=dict()
index_id=0
for i in tqdm(glob("F:/skypile_jieba/*")[:500]):
    index_id+=1

    one=pd.read_pickle(i,compression="zip")

    for _,v in one.items():
        for k in  v:
            voc[k]=voc.get(k,0)+1
    # print()

pd.to_pickle(voc,"hex_voc.pkl")